#---------------------------------------
#
#Script: cscope.py
#Description: runs cscope on every file and every revision of a given repository database, storing the data in sql
#
#Author: Alex Gourley (acgourley@ucdavis.edu)
#
#---------------------------------------

import re
import _pg, pg
import os
import sys
import revcalc
import datetime
import interface
import repository
        
class Function:
    "holds info about a function's name/location...no revision information, should not be stored long"
    def __init__(self, funcname, filename, start, end):
        self.funcName = funcname
        self.fileName = filename
        self.startLine = start
        self.endLine = end
        self.interface = [] #list of tokens
    def __print__(self):
        print "name:",self.funcName,"filename:", self.fileName, "lines:", self.startLine, "-", self.endLine, "interface:", self.interface
    def getInterfaceString(self):
        interfaceString = ''
        for t in self.interface:
            interfaceString = interfaceString + t + ' '
        return interfaceString
        
    def renameBasedOnTokens(self):
        "renames function class to the token to the left of the last closed set of parenthesis of the line of code"
        i = len(self.interface) - 1 #place i at last index
        level = 0 #depth in parenthesis
        while i >= 0:
            if self.interface[i] == ')':
                level += 1
            if self.interface[i] == '(':
                level -= 1
            i -= 1
            if level == 0 and i >= 0:
                self.funcName = self.interface[i]
                print self.interface
                print self.funcName
                break
        return  
        
def runCscope(path):    
    s = "cd " + path + "\n rm cscope.out"
    print s
    os.system(s)
    s = "cd " + path + "\n cscope -c -R -b -k"
    print s
    os.system(s)
    

    
def getFunctions(dataFile):
    "returns a list of Function classes, parsed from a cscope file"
    
    #variables
    fileName = funcName = start = end = lastNumber = None
    functions = []
    errorlog = file('cscope.errorlog', 'w')
    
    number = re.compile("[0-9]")
    functionLine = re.compile("\t\$")
    startState, inFile, inFunction = range(3) 
    state = startState
    lastline = ''
    for line in dataFile:
        
        #match if file section start
        #in cscope.out all file sections start with <tab>@
        if re.match("\t@", line):
            line = re.sub("\t@|\n", '', line)
            #now line should be the filename
            fileName = line
            state = inFile
            continue
        
        elif number.match(line):
            #strip non numbers
            lastNumber = re.sub("\D",'', line)
            
        #loop till we find a blank line
        elif state == inFile:
            if functionLine.match(line):
                line = re.sub("\t\$|\n", '', line)
                #now line should be function name
                funcName = line
                start = lastNumber
                state = inFunction
                continue
        elif state == inFunction:
            #match if function def end
            if re.match("\t}", line):
                end = lastNumber
                functions.append(Function(funcName, fileName, int(start), int(end)))
                state = inFile
                continue
        #if we hit a function closing brace at any point while the state is not inFunction...record this. We missed a function. 
        if state != inFunction and re.match("\t}", line):
            print >>errorlog, "found unexpected end of func:", fileName, lastNumber
    errorlog.close()
    return functions


def getFileDataHash(db):
    "load the files + revisions from sql, return data in a hash RepFiles keyed by filename"
    
    try:
        pgo = db.query("select Filename, Revision from files where sourcefile = True ORDER BY datetime");
    except Error:
        print "error! could not get data from table commits"
    #stores data about what files we haven't mined data on yet
    fileDataHash = {}
    if(pgo != None):
        results = pgo.getresult()
        for result in results:
            filename = result[0]
            revision = result[1]
            if revision == '1.1.1.1':#an exception for the base CVS revision. Of course for SVN stuff we don't filter out the base revision for a file...so we should probably change the program logic to handle this problem later down the stream. 
                continue
            if(fileDataHash.has_key(filename)):
                fileDataHash[filename].addRev(revision)
            else:
                fileDataHash[filename] = repository.RepFile(filename)
                fileDataHash[filename].addRev(revision)
    else:
        print "Could not get commit data from database!"
        
    return fileDataHash


def storeFunctions(db, rep, log):
    "loops over the repository, storing functions"
        
    #toDoHash is a hash of files each with a complete set of revision numbers
    #we sorted the sql query date, so the last entry of each revision list is the final revision pair

    toDoHash = getFileDataHash(db)
    
    #backup cwd
    scriptPath = os.getcwd()
    
    rep.prepUpdate() #calls command to make every file in rep updatable
    
    count = 0
    #loop until we're out of rev's for the files in the hash 
    while True:
        #count how many layers of revs we haved peeled off
        count += 1
        delList = [] #list to store hash keys scheduled to be deleted
            
        #loop over all the items in the hash table: f->filename (key) and c->Commit class
        for f,c in toDoHash.iteritems():                    
            #update each file f with the rev at the end of the rev list
            rep.updateFileToRev(f, c.fileRevList[-1].finalRev)
        
        #if we deleted the last of them: end loop, we're out of stuff to do.
        if len(toDoHash.keys()) == 0:
            break
        
        #run CScope 
        runCscope(rep.path)
        
        #get function names, containing files, and start/end locations
        try:
            functions = getFunctions(open(rep.path + "cscope.out"))
        except IOError:
            
            for f,c in toDoHash.iteritems():
                print "file:", f
            print "Could not open cscope.out which means it did not see any source code."
            print "Above are the files we were waiting to process."
            sys.exit(2)
        
        #at this point, we do not yet have the interfaces for the functions. Furthermore, we are not sure if the functions are actually functions or if they are weird preprocessor cases that cscope picked up anyway. We will go ahead and inspect for the interface while also verifiying it at least resembles a function enough for the interface parser to read it.
        curFileName = functions[0].fileName
        curFunctions = []
        functionsWithInterfaces = []
        for f in functions:
            if(curFileName == f.fileName):
                curFunctions.append(f)
            else:
                print "parsing interfaces from:",curFileName, "on lines:"
                for a in curFunctions:
                    print a.startLine
                #getCInterfaceTokens takes in a file and list of functions, returns same list of functions with interface data filled in.
                functionsWithInterfaces.extend(interface.getCInterfaceTokens(file(rep.path + curFileName), curFunctions))
                
                curFileName = f.fileName
                curFunctions = [f]
        
        #print functions
        for f in functionsWithInterfaces:
            #change function name to token left of open paren
            f.renameBasedOnTokens()
            
            try:
                s = "insert into functions (name, startline, endline, FileID, interface) VALUES('"+ f.funcName +"',"+ str(f.startLine) +","+ str(f.endLine) +",(select FileID from Files where Revision = '"+ toDoHash[f.fileName].fileRevList[-1].finalRev +"' and Filename = '"+ f.fileName +"'),'" + f.getInterfaceString().encode("string_escape") + "')"
                db.query(s)
            except KeyError:
                print "filename:", f.fileName, "isn't in toDoHash (gathered from SQL Files Table), probably because cscope scoped a non source code file"
            except _pg.ProgrammingError, e:
                print e
                print s
                print f
                sys.exit()

                
        #loop all files, pop'ing the revision we just did.
        for f,c in toDoHash.iteritems():
            #last revision for this file            
            if len(c.fileRevList) == 1:
                delList.append(f)
                continue
            
            try:
                # the pop call means we have shortened revPairList a notch
                c.fileRevList.pop()
            except IndexError:
                #this file has no more revisions we need to log
                #so schedule it to be deleted after the iteration
                delList.append(f)
                continue
        
        #delete all the files that have no more revisions to do
        for i in delList:
            del toDoHash[i]
            #delete the file from the working copy to speed things up
            s = "cd " + rep.path + "\n rm -f " + i
            print s

    return

def getCallerFunctions(function, path):
    "takes in a function string, returns list of Function classes which called the passed in function"
    calledFuncs = set()
    scriptDir = os.getcwd()
    os.chdir(path)
    for line in os.popen("cscope -d -L -3 " + function).readlines():
        file, function, line, text = line.split(" ", 3)
        calledFuncs.add(Function(function, file, 0,0))
    os.chdir(scriptDir)
    return calledFuncs
    
def calcCallGraph(db, rep, date):

    "stores a call graph of the current source code in the codePath"
    
    scriptPath = os.getcwd()

    runCscope(rep.path)
    
    try:
        cscopeFile = open(rep.path + "cscope.out")
    except IOError:
        print "Could not open Cscope file, no source found."
        return
    
    targetFunctions = getFunctions(cscopeFile)
    for func in targetFunctions:
        callerFunctions = getCallerFunctions(func.funcName, rep.path)
        for callerFunc in callerFunctions:
            #print "\n",callerFunc.fileName, "::", callerFunc.funcName, "calls", func.fileName, "::", func.funcName, "at", date.isoformat()
            #we need to look up the fileid of the caller
            s = "select fileid from files where filename = '" + callerFunc.fileName + "' and datetime < '" + date.isoformat() + "' order by datetime desc limit 1"
            callerFileID = db.query(s).getresult()
            if callerFileID == []:
                print "***ERROR: no file:", callerFunc.fileName, "in database at the given date, check Files table validity."
                continue
            callerFileID = callerFileID[0][0]
            #print "**At date:", date.isoformat(), "and in file:", callerFunc.fileName, "caller fileid is:", callerFileID
            
            #now we want to find the caller funcid
            s = "select funcid from functions where fileid = " + str(callerFileID) + " and name = '" + callerFunc.funcName + "'"
            callerFuncID = db.query(s).getresult()
            if callerFuncID == []:
                print "***",s
                print "***ERROR: No caller function:", callerFunc.funcName, "in fileid:", callerFileID
                continue
            callerFuncID = callerFuncID[0][0]
            #print "**the funcion caller is:", callerFunc.funcName, "with id:", callerFuncID
            
            #now we want the target file id
            s = "select fileid from files where filename = '" + func.fileName + "' and datetime < '" + date.isoformat() + "' order by datetime desc limit 1"
            targetFileID = db.query(s).getresult()
            if targetFileID == []:
                print "***No target File:", func.fileName, "before date:", date.isoformat()
                #sys.exit()
                continue
            targetFileID = targetFileID[0][0]
            #print "**The file called is:", func.fileName, "at date:", date.isoformat(), "with id:", targetFileID
            s = "select funcid from functions where fileid = " + str(targetFileID) + " and name = '" + func.funcName + "'"
            print "**",s
            targetFuncID = db.query(s).getresult()
            if(targetFuncID == []):
                #this means the function was not found in the database. It is most likely a system call.
                print "***** Target function:", func.funcName, "was not in the project"
                continue
            targetFuncID = targetFuncID[0][0]
            s = "INSERT INTO calls (callerfuncid, targetfuncid, date) values(" + str(callerFuncID) +"," + str(targetFuncID) + ",'" + date.isoformat() + "')"
            print "**",s
            db.query(s)
    return



#just pasted this code in here, should change calcMonthlyCallGraph to make use of this...will be easier to change the date segments in the future
def getMonthList(firstDate, lastDate):
    "takes in a start and end datetime, returning a list of datetimes (start, one month, two months, ... , last month before lastDate)"     
    dates = []
    currentDate = firstDate
    while(currentDate < lastDate):
        dates.append(currentDate)
        currentDate = currentDate + datetime.timedelta(30)
    return dates
    
def calcMonthlyCallGraphs(db, rep, log):
    #determine first and last dates where the rep was modified
    s = "select datetime from files order by datetime"
    firstDateString = db.query(s).getresult()[0][0]
    print "first datetime is:", firstDateString
    
    s = "select datetime from files order by datetime desc"
    lastDateString = db.query(s).getresult()[0][0]
    print "last datetime is:", lastDateString
    
    #create python date objects from sql date strings, ex date: 1999-06-18 *timenumbers*
    firstDate = datetime.date(int(firstDateString[:4]), int(firstDateString[5:7]), int(firstDateString[8:10]))
    lastDate = datetime.date(int(lastDateString[:4]), int(lastDateString[5:7]), int(lastDateString[8:10]))
    
    #lets start at the point 30 days ahead of the first file commit
    currentDate = firstDate + datetime.timedelta(30) #30 days after first repo commit
    rep.prepUpdate()
    while(currentDate < lastDate):
        print "running call graph analysis on:", currentDate.isoformat()
        rep.updateToDate(currentDate)
        calcCallGraph(db, rep, currentDate)
        currentDate = currentDate + datetime.timedelta(30)
    
    
    
